Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

AMORE Train/predict Functions

#library(AMORE)
hyperParams.AMORE <- function(optim_method, ...) {
    
    if (!is.element(optim_method, c("ADAPTgd", "ADAPTgdwm", "BATCHgd", "BATCHgdwm"))) stop("Invalid Parameters.")
    if (optim_method == "ADAPTgd") {iter <- maxit1storderB; lr <- 0.01; momentum <- 0; hidden_activation <- "tansig"} 
    if (optim_method == "ADAPTgdwm") {iter <- maxit1storderB; lr <- 0.01; momentum <- 0.8; hidden_activation <- "sigmoid"} 
    if (optim_method == "BATCHgd") {iter <- maxit1storderC; lr <- 0.01; momentum <- 0; hidden_activation <- "tansig"}
    if (optim_method == "BATCHgdwm") {iter <- maxit1storderC; lr <- 0.01; momentum <- 0.8; hidden_activation <- "tansig"}
    
    params <- paste0("method=", optim_method, "_lr=", lr, "_iter=", iter, "_momentum=", momentum, "_hidden_activation=", hidden_activation)
    
    out <- list(iter = iter, lr = lr, params = params, momentum = momentum, hidden_activation = hidden_activation, method = optim_method)
    
    return (out)
}

NNtrain.AMORE <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    iter <- hyper_params$iter
    lr <- hyper_params$lr
    momentum <- hyper_params$momentum
    hidden_activation <- hyper_params$hidden_activation
    method <- hyper_params$method
    
    net_structure <- AMORE::newff(n.neurons = c(ncol(x), hidden_neur, 1), 
                                  learning.rate.global = lr, 
                                  error.criterium = "LMS", 
                                  hidden.layer = hidden_activation, 
                                  method = method, 
                                  momentum.global = momentum)
    
    NNreg <- AMORE::train(net = net_structure, 
                          P = x, 
                          T = y, 
                          error.criterium = "LMS", 
                          report = FALSE, 
                          n.shows = iter, 
                          show.step = 1)
    return (NNreg)
}
NNpredict.AMORE <- function(object, x, ...)
    AMORE::sim.MLPnet(object$net, x)    
NNclose.AMORE <- function()
  if("package:AMORE" %in% search())
    detach("package:AMORE", unload=TRUE)
AMORE.method <- c("ADAPTgd", "ADAPTgdwm", "BATCHgd", "BATCHgdwm")
AMORE.prepareZZ <- list(xdmv = "d", ydmv = "v", zdm = "d", scale = TRUE)

if(FALSE)
res <- train_and_predict_1data(1, AMORE.method, "NNtrain.AMORE", "hyperParams.AMORE", "NNpredict.AMORE", 
                               NNsummary, "NNclose.AMORE", NA, AMORE.prepareZZ, nrep=2, echo=TRUE, 
                               doplot=FALSE, echoreport=0,
                               pkgname="AMORE", pkgfun="train", rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "AMORE", pkgfun = "train", AMORE.method,
  prepareZZ.arg = AMORE.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "ADAPTgd")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
ADAPTgd
RMSE MSE MAE WAE time
mDette 1.2271 1.5059 0.8825 4.9979 0.61
mFriedman 0.0256 0.0007 0.0207 0.0722 0.62
kable(apply(res[,,2,], c(3,1), min), caption = "ADAPTgdwm")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
ADAPTgdwm
RMSE MSE MAE WAE time
mDette 0.3844 0.1478 0.2961 1.5719 0.89
mFriedman 0.0408 0.0017 0.0303 0.1716 0.98
kable(apply(res[,,3,], c(3,1), min), caption = "BATCHgd")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
BATCHgd
RMSE MSE MAE WAE time
mDette 2.8763 8.2729 2.1004 15.4620 1.82
mFriedman 0.0827 0.0068 0.0627 0.2734 1.84
kable(apply(res[,,4,], c(3,1), min), caption = "BATCHgdwm")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
BATCHgdwm
RMSE MSE MAE WAE time
mDette 2.7630 7.6339 2.0657 14.2804 1.84
mFriedman 0.0903 0.0082 0.0705 0.2335 1.85

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

ANN2 Train/predict Functions - arguments x,y

#library(ANN2)
ANN2.method <- c("sgd", "adam", "rmsprop")
hyperParams.ANN2 <- function(optim_method, ...) {
    
    if (!is.element(optim_method, c("sgd", "adam", "rmsprop"))) stop("Invalid Parameters.")
    if (optim_method == "sgd")     { iter <- maxit1storderB; lr <- 0.01} 
    if (optim_method == "adam")    { iter <- maxit1storderB; lr <- 0.01} 
    if (optim_method == "rmsprop") { iter <- maxit1storderB; lr <- 0.01} 
    
    params <- paste0("method=", optim_method, "_", "lr=", lr, "_", "iter=", iter)
    
    out <- list(method = optim_method, iter = iter, lr = lr, params = params)
    
    return (out)
}

NNtrain.ANN2 <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...){
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    iter <- hyper_params$iter
    lr <- hyper_params$lr
    method <- hyper_params$method
    
    NNreg <- ANN2::neuralnetwork(X = x, y = y, 
                           val.prop = 0, 
                           standardize = FALSE, 
                           hidden.layers = hidden_neur, 
                           regression = TRUE,
                           loss.type = "squared",
                           n.epochs = iter,
                           optim.type = method,
                           learn.rates = lr,
                           verbose = FALSE,
                           random.seed = as.integer(runif(1)*10000000))
    
   return (NNreg)
}
NNpredict.ANN2 <- function(object, x, ...)
    as.numeric(predict(object, x)$predictions)    
NNclose.ANN2 <- function()
  if("package:ANN2" %in% search())
    detach("package:ANN2", unload=TRUE)
ANN2.prepareZZ <- list(xdmv = "d", ydmv = "v", zdm = "d", scale = TRUE)

if(FALSE)
res <- train_and_predict_1data(1, ANN2.method, "NNtrain.ANN2", "hyperParams.ANN2", "NNpredict.ANN2", 
                               NNsummary, "NNclose.ANN2", NA, ANN2.prepareZZ, nrep=2, echo=TRUE, doplot=FALSE,
                               pkgname="ANN2", pkgfun="neuralnetwork", rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "ANN2", pkgfun = "neuralnetwork", ANN2.method,
  prepareZZ.arg = ANN2.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "sgd")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
sgd
RMSE MSE MAE WAE time
mDette 0.3719 0.1383 0.2773 1.6586 0.98
mFriedman 0.0171 0.0003 0.0139 0.0546 1.04
kable(apply(res[,,2,], c(3,1), min), caption = "adam")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
adam
RMSE MSE MAE WAE time
mDette 0.4434 0.1966 0.3222 1.6566 1.03
mFriedman 0.0170 0.0003 0.0135 0.0516 1.14
kable(apply(res[,,3,], c(3,1), min), caption = "rmsprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
rmsprop
RMSE MSE MAE WAE time
mDette 0.4517 0.2040 0.3579 1.5754 1.01
mFriedman 0.0269 0.0007 0.0212 0.0832 1.08

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

automl Train/predict Function - arguments x,y

#library(automl)
automl.method <- c("trainwgrad", "trainwpso")
hyperParams.automl <- function(optim_method, ...) {
    
    if (!is.element(optim_method, c("trainwgrad", "trainwpso"))) stop("Invalid Parameters.")
    
    hidden_activation = "tanh"
    params <- paste0("method=", optim_method, "_", "hidden_activation=", hidden_activation)
    
    out <- list(hidden_activation = hidden_activation, params = params)
    
    return (out)
}
NNtrain.automl <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    hidden_activation <- hyper_params$hidden_activation
    # NNreg <- automl::automl_train(Xref = x, Yref = y)
    
    NNreg <- automl::automl_train_manual(Xref = x, Yref = y,
                                         hpar = list(modexec = optim_method,
                                                     layersshape = c(hidden_neur, 0),
                                                     layersacttype = c(hidden_activation, ""),
                                                     verbose = FALSE,
                                                     seed = as.integer(runif(1)*10000000)))

    return (NNreg)
}
NNpredict.automl <- function(object, x, ...)
    automl::automl_predict(model=object, X=x)
NNclose.automl <- function()
  if("package:automl" %in% search())
    detach("package:automl", unload=TRUE)
automl.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)

if(FALSE)
res <- train_and_predict_1data(1, automl.method, "NNtrain.automl", "hyperParams.automl", "NNpredict.automl", 
                               NNsummary, "NNclose.automl", NA, automl.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="automl", pkgfun="automl_train_manual", rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "automl", pkgfun = "automl_train_manual", automl.method,
  prepareZZ.arg = automl.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "trainwgrad")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
trainwgrad
RMSE MSE MAE WAE time
mDette 6.4332 41.3867 4.6633 28.6250 0.47
mFriedman 0.1444 0.0209 0.1123 0.4911 0.47
kable(apply(res[,,2,], c(3,1), min), caption = "trainwpso")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
trainwpso
RMSE MSE MAE WAE time
mDette 4.9447 24.4504 3.6018 23.5120 0.94
mFriedman 0.1312 0.0172 0.1010 0.4279 0.97

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/"))
{  
  odir <- "D:/GSoC2020/Results/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

BNN Train/predict Function - arguments x,y

#library(BNN)
BNN.method <- "none"
hyperParams.BNN <- function(optim_method, ...) {
    return(list(iter = maxit1storderD, popN = 10, nCPUs = 10))
}
NNtrain.BNN <- function(x, y, dataxy, formula, neur, optim_method, hyperParams,...) {
    
    hyper_params <- do.call(hyperParams.BNN, list(BNN.method, ...))
    
    NNreg <- BNN::BNNsel(X = x,Y = y,hid_num = neur,train_num = (nrow(dataxy)-1), 
                  total_iteration = hyper_params$iter, popN = hyper_params$popN, 
                  nCPUs = hyper_params$nCPUs)
    
    return (NNreg)
}
NNpredict.BNN <- function(object, x, ...)
    c(object$fit, object$pred)
NNclose.BNN <- function()
  if("package:BNN" %in% search())
    detach("package:BNN", unload=TRUE)
BNN.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, BNN.method, "NNtrain.BNN", "hyperParams.BNN", "NNpredict.BNN", 
                               NNsummary, "NNclose.BNN", NA, BNN.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="BNN", pkgfun="BNN", csvfile=TRUE, rdafile=TRUE, odir=odir)
if(FALSE)
res <- trainPredict_1mth1data(1, BNN.method[1], "NNtrain.BNN", "hyperParams.BNN", "NNpredict.BNN", 
                               NNsummary, BNN.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="BNN", pkgfun="BNN", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "BNN", pkgfun = "BNNsel", BNN.method,
  prepareZZ.arg = BNN.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 1.1127 1.2381 0.8611 5.5653 21.36
mFriedman 0.0236 0.0006 0.0190 0.0578 23.66
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 1.2390 1.5352 0.9145 6.7044 25.65
mFriedman 0.0322 0.0010 0.0248 0.0910 24.27

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder <- 200
maxit1storder <- 1000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

brnn Train/predict Function - arguments x,y

#library(brnn)
brnn.method <- "gaussNewton"
hyperParams.brnn <- function(optim_method, ...) {
    
    if (!is.element(optim_method, c("gaussNewton"))) stop("Invalid Parameters.")
    iter   <- maxit2ndorder
    
    params <- paste0("method=", optim_method, "_iter=", iter)
    
    out <- list(iter = iter, params = params)
    
    return (out)
}
NNtrain.brnn <- function(x, y, dataxy, formula, neur, optim_method, hyperParams,...) {
    
    hyper_params <- do.call(hyperParams.brnn, list(brnn.method))
    
    iter <- hyper_params$iter
    
    NNreg <- brnn::brnn(x, y, neur, normalize = FALSE, epochs = iter, verbose = FALSE)
    
    return (NNreg)
}
NNpredict.brnn <- function(object, x, ...)
    predict(object, x)
NNclose.brnn <- function()
  if("package:brnn" %in% search())
    detach("package:brnn", unload=TRUE)
brnn.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, brnn.method, "NNtrain.brnn", "hyperParams.brnn", "NNpredict.brnn", 
                               NNsummary, "NNclose.brnn", NA, brnn.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="brnn", pkgfun="brnn", csvfile=TRUE, rdafile=TRUE, odir=odir)
if(FALSE)
res <- trainPredict_1mth1data(1, brnn.method[1], "NNtrain.brnn", "hyperParams.brnn", "NNpredict.brnn", 
                               NNsummary, brnn.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="brnn", pkgfun="brnn", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "brnn", pkgfun = "brnn", brnn.method,
  prepareZZ.arg = brnn.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.2291 0.0525 0.1833 0.8047 0.20
mFriedman 0.0046 0.0000 0.0037 0.0139 0.15
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 2.0510 4.2067 1.46275 12.41105 0.205
mFriedman 0.0046 0.0000 0.00370 0.01400 0.270

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder <- 200
maxit1storder <- 1000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

CaDENCE Train/predict Function - arguments x,y

#library(CaDENCE)
CaDENCE.method <- c("optim", "psoptim", "Rprop")
hyperParams.CaDENCE <- function(optim_method, ...) {
  
    if (optim_method == "optim")    {iter <- maxit2ndorder} 
    if (optim_method == "psoptim")  {iter <- maxit1storder}
    if (optim_method == "Rprop")    {iter <- maxit1storder}
  
    params <- paste0("method=", optim_method, "_iter=", iter)
    
    out <- list(iter = iter, method = optim_method, params = params, maxit.Nelder=1)
    return (out)
}

NNtrain.CaDENCE <- function(x, y, dataxy, formula, neur, optim_method, hyperParams,...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    iter <- hyper_params$iter
    method <- hyper_params$method

    NNreg <- CaDENCE::cadence.fit(x = x, y = y, 
                                iter.max = iter, 
                                n.hidden = neur, 
                                hidden.fcn = tanh, 
                                method = method, 
                                n.trials = 1, 
                                trace = 0, 
                                maxit.Nelder = 1, 
                                f.cost = cadence.cost,
                                distribution = list(density.fcn = dnorm,
                                                    parameters = c("mean", "sd"),
                                                    parameters.fixed = NULL,
                                                    output.fcns = c(identity, exp)))
    return (NNreg)
}
NNpredict.CaDENCE <- function(object, x, ...)
    CaDENCE::cadence.predict(x = x, fit = object)[,1]
NNclose.CaDENCE <- function()
  if("package:CaDENCE" %in% search())
    detach("package:CaDENCE", unload=TRUE)
CaDENCE.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)

if(FALSE)
res <- trainPredict_1data(1, CaDENCE.method, "NNtrain.CaDENCE", "hyperParams.CaDENCE", "NNpredict.CaDENCE", 
                               NNsummary, "NNclose.CaDENCE", NA, CaDENCE.prepareZZ, nrep=2, echo=TRUE, doplot=FALSE,
                               pkgname="CaDENCE", pkgfun="cadence.fit", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "CaDENCE", pkgfun = "cadence.fit", CaDENCE.method,
  prepareZZ.arg = CaDENCE.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = TRUE)

Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "optim")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
optim
RMSE MSE MAE WAE time
mDette 0.4785 0.2289 0.3233 2.3171 7.03
mFriedman 0.0205 0.0004 0.0154 0.0603 9.11
kable(apply(res[,,2,], c(3,1), min), caption = "psoptim")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
psoptim
RMSE MSE MAE WAE time
mDette 3.4564 11.9469 1.8714 20.3055 11.22
mFriedman 0.0944 0.0089 0.0629 0.3260 12.01
kable(apply(res[,,3,], c(3,1), min), caption = "Rprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
Rprop
RMSE MSE MAE WAE time
mDette 1.3246 1.7545 0.6774 10.0693 17.10
mFriedman 0.0331 0.0011 0.0223 0.1250 22.41

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

caret Train/predict Functions - arguments x,y

#library(caret)
caret.method <- "none"
hyperParams.caret <- function(...) {
    return (list(iter=maxit2ndorder, trace=FALSE))
}
NNtrain.caret <- function(x, y, dataxy, formula, neur, method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(...))
    
    caret::avNNet(x, y, repeats = 3, size = neur, linout = TRUE, trace=hyper_params$trace, bag = TRUE)
}
NNpredict.caret <- function(object, x, ...)
    predict(object, newdata=x)
NNclose.caret <- function()
  if("package:caret" %in% search())
    detach("package:caret", unload=TRUE)
caret.prepareZZ <- list(xdmv = "d", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, caret.method, "NNtrain.caret", "hyperParams.caret", "NNpredict.caret", 
                               NNsummary, "NNclose.caret", NA, caret.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="caret", pkgfun="caret", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "caret", pkgfun = "avNNet", caret.method,
  prepareZZ.arg = caret.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.3949 0.1559 0.2735 2.3692 0.09
mFriedman 0.0327 0.0011 0.0276 0.0940 0.14
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.8183 0.66965 0.58715 4.8147 0.12
mFriedman 0.0569 0.00325 0.04370 0.1517 0.16

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

deepdive Train/predict Function - arguments x,y

#library(deepdive)
deepdive.method <- c("gradientDescent","momentum","rmsProp","adam")
hyperParams.deepdive <- function(optim_method, ...) {
    
    if (optim_method == "gradientDescent")  {maxiter <- maxit1storderB} 
    if (optim_method == "momentum")         {maxiter <- maxit1storderB}
    if (optim_method == "rmsProp")          {maxiter <- maxit1storderA}
    if (optim_method == "adam")             {maxiter <- maxit1storderA}
    
    params <- paste0("method=", optim_method, "_iter=", maxiter)
  
    out <- list(iter = maxiter, method = optim_method, modelType = "regress", eta = 0.8, print = 1000, params)
    return (out)
}
NNtrain.deepdive <- function(x, y, dataxy, formula, neur, optim_method, hyperParams, NNfullformula, NNparam,...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    NNreg <- deepdive::deepnet(x = x, y = y, c(neur), modelType = hyper_params$modelType,
                     iterations = hyper_params$iter, eta=hyper_params$eta, 
                     optimiser=hyper_params$method, printItrSize=hyper_params$print)
    
    return (NNreg)
}
NNpredict.deepdive <- function(object, x, ...)
  predict.deepnet(object,newData=x)$ypred
NNclose.deepdive <- function()
  if("package:deepdive" %in% search())
    detach("package:deepdive", unload=TRUE)
deepdive.prepareZZ <- list(xdmv = "d", ydmv = "d", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, deepdive.method, "NNtrain.deepdive", "hyperParams.deepdive", "NNpredict.deepdive", 
                               NNsummary, "NNclose.deepdive", NA, deepdive.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="deepdive", pkgfun="ann", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "deepdive", pkgfun = "deepnet", deepdive.method,
  prepareZZ.arg = deepdive.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

kable(apply(res[,,1,], c(3,1), min), caption = "gradientDescent")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
gradientDescent
RMSE MSE MAE WAE time
mDette 0 0 0 -Inf 0.75
mFriedman 0 0 0 -Inf 0.75
kable(apply(res[,,2,], c(3,1), min), caption = "momentum")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
momentum
RMSE MSE MAE WAE time
mDette 7.6568 58.6271 6.1862 29.3157 3.74
mFriedman 0.1467 0.0215 0.1153 0.4638 3.92
kable(apply(res[,,3,], c(3,1), min), caption = "rmsProp")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
rmsProp
RMSE MSE MAE WAE time
mDette 7.0680 49.9572 5.7558 31.0085 0.73
mFriedman 0.1319 0.0174 0.1013 0.3916 0.76
kable(apply(res[,,4,], c(3,1), min), caption = "adam")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
adam
RMSE MSE MAE WAE time
mDette 3.8791 15.0473 2.8228 21.0823 0.77
mFriedman 0.1098 0.0121 0.0876 0.3394 0.79

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

deepnet Train/predict Function - arguments x,y

#library(deepnet)
deepnet.method <- "gradientDescent"
hyperParams.deepnet <- function(optim_method, ...) {
    
    if (!is.element(optim_method, c("gradientDescent"))) stop("Invalid Parameters.")

    iter <- maxit1storderC
    lr <- 0.8
    dropout <- 0
    momentum <- 0.95
    hidden_activation <- "sigm"
    
    params <- paste0("method=", optim_method, "_", "iter=", iter, "_", "lr=", lr, "_", "dropout=", dropout, "_", "momentum=", momentum)
    
    out <- list(iter = iter, lr = lr, momentum = momentum, hidden_activation = hidden_activation, dropout = dropout, params = params)
    
    return (out)
}

NNtrain.deepnet <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    iter <- hyper_params$iter
    lr <- hyper_params$lr
    dropout <- hyper_params$dropout
    momentum <- hyper_params$momentum
    hidden_activation <- hyper_params$hidden_activation
    dropout <- hyper_params$dropout

    NNreg <- deepnet::nn.train(x = x, y = y, 
                               hidden = c(hidden_neur), 
                               activationfun = hidden_activation, 
                               learningrate = lr, 
                               output = 'linear', 
                               numepochs = iter, 
                               hidden_dropout = dropout, 
                               momentum = momentum)
    return (NNreg)
}
NNpredict.deepnet <- function(object, x, ...)
    deepnet::nn.predict(nn = object, x = x)
NNclose.deepnet <- function()
  if("package:deepnet" %in% search())
    detach("package:deepnet", unload=TRUE)
deepnet.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- train_and_predict_1data(1, CaDENCE.method, "NNtrain.deepnet", "hyperParams.deepnet", "NNpredict.deepnet", 
                               NNsummary, "NNclose.deepnet", NA, deepnet.prepareZZ, nrep=2, echo=TRUE, doplot=FALSE,
                               pkgname="deepnet", pkgfun="nn.train", rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "deepnet", pkgfun = "nn.train", deepnet.method,
  prepareZZ.arg = deepnet.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.4944 0.2444 0.3768 1.8511 10.68
mFriedman 0.0609 0.0037 0.0498 0.1731 10.87
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.5389 0.2904 0.4143 2.2999 10.73
mFriedman 0.0884 0.0078 0.0778 0.2150 10.88

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

elmNNRcpp Train Function - arguments x,y

#library(elmNNRcpp)
elmNNRcpp.method <- "extremeML"
hyperParams.elmNNRcpp <- function(optim_method, ...) {
    
    if (!is.element(optim_method, c("extremeML"))) stop("Invalid Parameters.")

    moorep_pseudoinv_tol <- 0.01
    wt_init <- "normal_gaussian"
    hidden_activation <- "tansig"
    
    params <- paste0("method=", optim_method, "_wtinit=", wt_init, "_hidden_activation", hidden_activation)
    
    out <- list(wt_init = wt_init, hidden_activation = hidden_activation, 
                params = params, moorep_pseudoinv_tol=moorep_pseudoinv_tol)
    
    return (out)
}
NNtrain.elmNNRcpp <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    wt_init <- hyper_params$wt_init
    hidden_activation <- hyper_params$hidden_activation
    moorep_pseudoinv_tol <- hyper_params$moorep_pseudoinv_tol

    NNreg <- elmNNRcpp::elm_train(x, y, 
                                  nhid = hidden_neur, 
                                  actfun=hidden_activation, 
                                  init_weights = wt_init, 
                                  bias = TRUE, 
                                  moorep_pseudoinv_tol = moorep_pseudoinv_tol, 
                                  verbose = FALSE,
                                  seed = as.integer(runif(1)*10000000))
    return (NNreg)
}
NNpredict.elmNNRcpp <- function(object, x, ...)
    elmNNRcpp::elm_predict(elm_train_object = object, newdata = x, normalize = FALSE)
NNclose.elmNNRcpp <- function()
  if("package:elmNNRcpp" %in% search())
    detach("package:elmNNRcpp", unload=TRUE)
elmNNRcpp.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- train_and_predict_1data(1, elmNNRcpp.method, "NNtrain.elmNNRcpp", "hyperParams.elmNNRcpp", "NNpredict.elmNNRcpp", 
                               NNsummary, "NNclose.elmNNRcpp", NA, elmNNRcpp.prepareZZ, nrep=2, echo=TRUE, doplot=FALSE,
                               pkgname="elmNNRcpp", pkgfun="elm_train", rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "elmNNRcpp", pkgfun = "elm_train", elmNNRcpp.method,
  prepareZZ.arg = elmNNRcpp.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 7.1450 51.0506 5.3594 29.6591 0
mFriedman 0.1509 0.0228 0.1142 0.5290 0
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 7.6521 58.5542 6.0157 33.5874 0
mFriedman 0.1941 0.0377 0.1539 0.5902 0

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

ELMR Train Function - arguments formula,data

ELMR.method <- "extremeML"
hyperParams.ELMR <- function(optim_method, ...) {
    
    if (!is.element(optim_method, c("extremeML"))) stop("Invalid Parameters.")

    hidden_activation <- "sig"
    size_first_block <- 30
    size_each_chunk <- 30
    params <- paste0("method=", optim_method, "_hidden_activation=", hidden_activation)
    out <- list(hidden_activation = hidden_activation, params = params, 
                size_first_block=size_first_block, size_each_chunk=size_each_chunk)
    return (out)
}
NNtrain.ELMR <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    hidden_activation <- hyper_params$hidden_activation
    size_each_chunk <- hyper_params$size_each_chunk
    size_first_block <- hyper_params$size_first_block
    
    #OSelm_train.formula() call OSelm_training()
    NNreg <- ELMR::OSelm_train.formula(formula = formula,
                                       data = dataxy,
                                       Elm_type = "regression",
                                       nHiddenNeurons = hidden_neur,
                                       ActivationFunction = hidden_activation,
                                       N0 = size_first_block, Block = size_each_chunk)

    return (NNreg)
}
NNpredict.ELMR <- function(object, x, xy)
    ELMR::predict_elm(model = object, test = xy)$predicted
NNclose.ELMR <- function()
  if("package:ELMR" %in% search())
    detach("package:ELMR", unload=TRUE)
ELMR.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- train_and_predict_1data(1, ELMR.method, "NNtrain.ELMR", "hyperParams.ELMR", "NNpredict.ELMR", 
                               NNsummary, "NNclose.ELMR", NA, ELMR.prepareZZ, nrep=2, echo=TRUE, doplot=FALSE,
                               pkgname="ELMR", pkgfun="OSelm_train", rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "ELMR", pkgfun = "OSelm_train.formula", ELMR.method,
  prepareZZ.arg = ELMR.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 7.3740 54.3753 6.0069 27.6665 0
mFriedman 0.1646 0.0271 0.1334 0.4377 0
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 7.5131 56.4465 6.0818 30.7188 0.01
mFriedman 0.1781 0.0317 0.1451 0.4816 0.01

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

EnsembleBase Train/predict Function - arguments x,y

#library(EnsembleBase)
EnsembleBase.method <- "none"
hyperParams.EnsembleBase <- function(optim_method, ...) {
    out <- list(iter = maxit2ndorder, decay = 0)
    return (out)
}
NNtrain.EnsembleBase <- function(x, y, dataxy, formula, neur, optim_method, hyperParams,...) {
    
    hyper_params <- do.call(hyperParams.EnsembleBase, list(EnsembleBase.method))
    
    iter <- hyper_params$iter ; decay <- hyper_params$decay
    
    NNreg <- EnsembleBase::Regression.Batch.Fit(make.configs("nnet", config.df = expand.grid(decay=decay,size=c(neur),maxit=iter)), formula, dataxy, ncores = 1)
    
    return (NNreg)
}
NNpredict.EnsembleBase <- function(object, x, ...)
    predict(object, x)
NNclose.EnsembleBase <- function()
  if("package:EnsembleBase" %in% search())
    detach("package:EnsembleBase", unload=TRUE)
EnsembleBase.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, EnsembleBase.method, "NNtrain.EnsembleBase", "hyperParams.EnsembleBase", "NNpredict.EnsembleBase", 
                               NNsummary, "NNclose.EnsembleBase", NA, EnsembleBase.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="EnsembleBase", pkgfun="EnsembleBase", csvfile=TRUE, rdafile=TRUE, odir=odir)
if(FALSE)
res <- trainPredict_1mth1data(1, EnsembleBase.method[1], "NNtrain.EnsembleBase", "hyperParams.EnsembleBase", "NNpredict.EnsembleBase", 
                               NNsummary, EnsembleBase.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="EnsembleBase", pkgfun="EnsembleBase", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "EnsembleBase", pkgfun = "Regression.Batch.Fit", EnsembleBase.method,
  prepareZZ.arg = EnsembleBase.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.8921 0.7958 0.6648 3.978 0.00
mFriedman 0.0241 0.0006 0.0158 0.128 0.08
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 1.71410 2.95375 1.3691 7.0432 0.07
mFriedman 0.02725 0.00075 0.0189 0.1354 0.09

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

GMDHreg Train/predict Function - arguments x,y

#library(GMDHreg)
GMDHreg.method <- "none"
hyperParams.GMDHreg <- function(optim_method, ...) {
    return (list(G = 2, criteria = "PRESS"))
}
NNtrain.GMDHreg <- function(x, y, dataxy, formula, neur, optim_method, hyperParams,...) {
    
    hyper_params <- do.call(hyperParams.GMDHreg, list(GMDHreg.method))
    
    G <- hyper_params$G ; criteria <- hyper_params$criteria
    
    NNreg <- GMDHreg::gmdh.combi(X = x, y = y, G = G, criteria = criteria)
    
    return (NNreg)
}
NNpredict.GMDHreg <- function(object, x, ...)
    predict(object, x)
NNclose.GMDHreg <- function()
  if("package:GMDHreg" %in% search())
    detach("package:GMDHreg", unload=TRUE)
GMDHreg.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, GMDHreg.method, "NNtrain.GMDHreg", "hyperParams.GMDHreg", "NNpredict.GMDHreg", 
                               NNsummary, "NNclose.GMDHreg", NA, GMDHreg.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="GMDHreg", pkgfun="GMDHreg", csvfile=TRUE, rdafile=TRUE, odir=odir)
if(FALSE)
res <- trainPredict_1mth1data(1, GMDHreg.method[1], "NNtrain.GMDHreg", "hyperParams.GMDHreg", "NNpredict.GMDHreg", 
                               NNsummary, GMDHreg.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="GMDHreg", pkgfun="GMDHreg", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "GMDHreg", pkgfun = "gmdh.combi", GMDHreg.method,
  prepareZZ.arg = GMDHreg.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 2.1304 4.5385 1.7081 8.6036 0.1
mFriedman 0.0933 0.0087 0.0763 0.2012 410.9
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 2.1304 4.5385 1.7081 8.6036 0.11
mFriedman 0.0933 0.0087 0.0763 0.2012 413.53

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

h2o Train Function - arguments xy,y

#library(h2o)
h2o.method <- "gradientDescent"
hyperParams.h2o <- function(optim_method, ...) {
    
    if (!is.element(optim_method, c("gradientDescent"))) stop("Invalid Parameters.")
    
    hidden_activation = "Tanh"
    iter <- maxit1storderC
    rate <- 0.01
    stopping_rounds <- 500
    stopping_tolerance <- 1e-5
    distribution <- "gaussian"
    params <- paste0("method=", optim_method, "_", "hidden_activation=", hidden_activation)
    
    out <- list(hidden_activation = hidden_activation, iter = iter, params = params,
                rate=rate, stopping_rounds=stopping_rounds, stopping_tolerance=stopping_tolerance,
                distribution=distribution)
    
    return (out)
}
NNtrain.h2o <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    hidden_activation <- hyper_params$hidden_activation
    iter <- hyper_params$iter
    rate <- hyper_params$rate
    stopping_rounds <- hyper_params$stopping_rounds
    stopping_tolerance <- hyper_params$stopping_tolerance
    distribution <- hyper_params$distribution
    if(class(dataxy) != "H2OFrame")
      dataxy <- h2o::as.h2o(dataxy)

    NNreg <-   h2o::h2o.deeplearning(y = "y",
                                     training_frame = dataxy,
                                     overwrite_with_best_model = TRUE, 
                                     standardize = FALSE,
                                     activation = hidden_activation,
                                      adaptive_rate = TRUE,
                                     #rate = rate,
                                     hidden = hidden_neur,
                                     epochs = iter,
                                     train_samples_per_iteration = -1,
                                     initial_weight_distribution = "Normal",
                                     initial_weight_scale = 0.1,
                                     loss = "Quadratic",
                                     distribution = distribution,
                                     stopping_rounds = stopping_rounds,
                                     stopping_metric = "RMSE",
                                     stopping_tolerance = stopping_tolerance,
                                     seed = as.integer(runif(1)*10000000),
                                     verbose = FALSE
                                     )
    return (NNreg)
}
NNpredict.h2o <- function(object, x, ...)
{
  predictions <- h2o::h2o.predict(object, newdata=h2o::as.h2o(x))
  as.data.frame(predictions)$predict
}
NNclose.h2o <- function()
{
  h2o::h2o.shutdown(FALSE)
  if("package:h2o" %in% search())
    detach("package:h2o", unload=TRUE)
}
NNstart.h2o <- function()
{
  require("h2o", character.only = TRUE)
  h2o::h2o.init()
  h2o::h2o.no_progress()
}
h2o.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)

if(FALSE)
res <- train_and_predict_1data(1, h2o.method, "NNtrain.h2o", "hyperParams.h2o", "NNpredict.h2o", 
                               NNsummary, "NNclose.h2o", "NNstart.h2o", h2o.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="h2o", pkgfun="deeplearning", rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "h2o", pkgfun = "h2o.deeplearning", h2o.method,
  prepareZZ.arg = h2o.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.3159 0.0998 0.2385 1.1422 5.53
mFriedman 0.0170 0.0003 0.0133 0.0489 6.46
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.3494 0.1221 0.2792 1.3385 5.56
mFriedman 0.0181 0.0003 0.0142 0.0688 6.55

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-   200
maxit1storderA <-  1000
maxit1storderB <-  5000
maxit1storderC <- 10000
maxit1storderD <- 50000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

keras Train Function - arguments x,y

#library(keras)
keras.method <- c("adam", "rmsprop", "sgd", "adadelta", "adagrad")
hyperParams.keras <- function(optim_method, ...) {
    
    if (!is.element(optim_method, c("adam", "rmsprop", "sgd", "adagrad", "adadelta"))) 
      stop("Invalid Parameters.")
    
    hidden_activation = "tanh"
    iter <- maxit1storderC
    lr <- 0.1
    
    params <- paste0("method=", optim_method, "_iter=", iter, "_lr=", lr, "_hidden_activation=", hidden_activation)
    
    out <- list(hidden_activation = hidden_activation, iter = iter, lr = lr, params = params)
    
    return (out)
}

NNtrain.keras <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, ...) {
        
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    iter <- hyper_params$iter
    
    early_stop <- callback_early_stopping(monitor = "loss", patience = 20, restore_best_weights = TRUE, mode = "auto", min_delta = 0.0001)
    #should we have a higher min_delta
    
    hidden_activation <- hyper_params$hidden_activation
    lr <- hyper_params$lr
    
    if (optim_method == "adam")     { op <- optimizer_adam(lr = lr)} 
    if (optim_method == "rmsprop")  { op <- optimizer_rmsprop(lr = lr)}
    if (optim_method == "adagrad")  { op <- optimizer_adagrad(lr = lr)}
    if (optim_method == "adadelta") { op <- optimizer_adadelta(lr = lr)}
    if (optim_method == "sgd")      { op <- optimizer_sgd(lr = lr)}

    model <- keras_model_sequential() %>%
        layer_dense(units = hidden_neur, activation = hidden_activation, input_shape = ncol(x)) %>%
        layer_dense(units = 1)

    model %>% compile(
        loss = "mse",
        optimizer = op,
        metrics = list("mean_absolute_error")
    )

    historylog <- model %>% fit(x, y, epochs = iter, verbose = 0, callbacks = list(early_stop))

    return (model)
}  

NNpredict.keras <- function(object, x, ...)
{
  object %>% predict(x)
}
NNclose.keras <- function()
{
  if("package:keras" %in% search())
    detach("package:keras", unload=TRUE)
}

keras.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)

if(FALSE)
res <- train_and_predict_1data(1, keras.method, "NNtrain.keras", "hyperParams.keras", "NNpredict.keras", 
                               NNsummary, "NNclose.keras", NA, keras.prepareZZ, nrep=2, echo=TRUE, doplot=FALSE,
                               pkgname="keras", pkgfun="fit", rdafile=TRUE, odir=odir, echoreport=2)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "keras", pkgfun = "fit", keras.method,
  prepareZZ.arg = keras.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "adam")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
adam
RMSE MSE MAE WAE time
mDette 0.6421 0.4123 0.4908 3.6562 2.38
mFriedman 0.0797 0.0063 0.0646 0.2525 2.31
kable(apply(res[,,2,], c(3,1), min), caption = "rmsprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
rmsprop
RMSE MSE MAE WAE time
mDette 2.9200 8.5264 2.0955 14.7276 1.6
mFriedman 0.1139 0.0130 0.0870 0.3589 1.5
kable(apply(res[,,3,], c(3,1), min), caption = "sgd")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
sgd
RMSE MSE MAE WAE time
mDette 0.6922 0.4792 0.5192 4.7577 4.62
mFriedman 0.0339 0.0011 0.0267 0.1058 4.19
kable(apply(res[,,4,], c(3,1), min), caption = "adadelta")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
adadelta
RMSE MSE MAE WAE time
mDette 1.0137 1.0275 0.6992 6.1293 196.76
mFriedman 0.0322 0.0010 0.0251 0.1097 92.81
kable(apply(res[,,5,], c(3,1), min), caption = "adagrad")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
adagrad
RMSE MSE MAE WAE time
mDette 0.7311 0.5344 0.5209 3.9791 10.03
mFriedman 0.0280 0.0008 0.0220 0.0936 11.50

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder <- 200
maxit1storder <- 1000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

MachineShop Train/predict Function - arguments formula, data

#library(MachineShop)
MachineShop.method <- "none"
hyperParams.MachineShop <- function(...) {
    return (list(iter=maxit2ndorder, trace=FALSE, linout=TRUE))
}
NNtrain.MachineShop <- function(x, y, dataxy, formula, neur, method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(...))
    trace <- hyper_params$trace
    maxit <- hyper_params$iter
    linout <- hyper_params$linout #linearoutpputunit
    myNN <- MachineShop::NNetModel(size = neur, linout = linout, maxit = maxit,
                                   trace=trace)
    MachineShop::fit(formula, data = dataxy, model = myNN)
    
}
NNpredict.MachineShop <- function(object, x, ...)
    as.numeric(predict(object, newdata=x, type="response"))
NNclose.MachineShop <- function()
  if("package:MachineShop" %in% search())
    detach("package:MachineShop", unload=TRUE)
MachineShop.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, MachineShop.method, "NNtrain.MachineShop", "hyperParams.MachineShop", "NNpredict.MachineShop", 
                               NNsummary, "NNclose.MachineShop", NA, MachineShop.prepareZZ, nrep=5,
                               echo=TRUE, doplot=FALSE, echoreport=0,
                               pkgname="MachineShop", pkgfun="fit", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "MachineShop", pkgfun = "fit", MachineShop.method,
  prepareZZ.arg = MachineShop.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.2756 0.0759 0.2151 1.1389 0.07
mFriedman 0.0098 0.0001 0.0079 0.0312 0.08
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.4381 0.19235 0.33275 1.83380 0.080
mFriedman 0.0324 0.00105 0.02560 0.11545 0.095

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder <- 200
maxit1storder <- 1000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

minpack.lm train/predict functions - arguments formula,data

#library(minpack.lm)
minpack.lm.method <- "none"
hyperParams.minpack.lm <- function(...) {
    return (list(iter=maxit2ndorder, sdnormstart=0.1))
}
NNtrain.minpack.lm <- function(x, y, dataxy, formula, neur, method, hyperParams, NNfullformula, NNparam, ...) {
    
    hyper_params <- do.call(hyperParams, list(...))
    
    start <- round(rnorm(NNparam, sd = hyper_params$sdnormstart), 4)
    names(start)  <- paste0("b", 1:NNparam)
    minpack.lm::nlsLM(NNfullformula, data = dataxy, start=start,
                      control = list(maxiter = hyper_params$iter))
}
NNpredict.minpack.lm <- function(object, x, ...)
  predict(object, newdata=as.data.frame(x))
NNclose.minpack.lm <- function()
  if("package:minpack.lm" %in% search())
    detach("package:minpack.lm", unload=TRUE)
minpack.lm.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, minpack.lm.method, "NNtrain.minpack.lm", "hyperParams.minpack.lm", "NNpredict.minpack.lm", 
                               NNsummary, "NNclose.minpack.lm", NA, minpack.lm.prepareZZ, nrep=5,
                               echo=TRUE, doplot=FALSE, echoreport=0,
                               pkgname="minpack.lm", pkgfun="nlsLM", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "minpack.lm", pkgfun = "nlsLM", minpack.lm.method,
  prepareZZ.arg = minpack.lm.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.1062 0.0113 0.0845 0.4569 0.23
mFriedman 0.0045 0.0000 0.0036 0.0131 0.36
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 1.07765 1.32760 0.91035 4.01865 0.235
mFriedman 0.05065 0.00345 0.04330 0.10965 0.390

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder <- 200
maxit1storder <- 1000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

monmlp train/predict functions - arguments x,y

#library(monmlp)
monmlp.method <- c("BFGS", "Nelder-Mead")
hyperParams.monmlp <- function(optim_method, ...) {
    if (optim_method == "BFGS")         {maxiter <- maxit2ndorder}
    if (optim_method == "Nelder-Mead")  {maxiter <- maxit1storder} 
    return (list(iter=maxiter, silent=TRUE, scale=TRUE, method = optim_method))
}
NNtrain.monmlp <- function(x, y, dataxy, formula, neur, optim_method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    iter <- hyper_params$iter
    method <- hyper_params$method
    scale <- hyper_params$scale
    silent <- hyper_params$silent
    
    monmlp::monmlp.fit(x, y, hidden1 = neur, scale.y = scale, silent=silent,
                         method = method, iter.max = iter)
}
NNpredict.monmlp <- function(object, x, ...)
  as.numeric(monmlp::monmlp.predict(x, weights=object))
NNclose.monmlp <- function()
  if("package:monmlp" %in% search())
    detach("package:monmlp", unload=TRUE)
monmlp.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, monmlp.method, "NNtrain.monmlp", "hyperParams.monmlp", "NNpredict.monmlp", 
                               NNsummary, "NNclose.monmlp", NA, monmlp.prepareZZ, nrep=2,
                               echo=TRUE, doplot=FALSE, echoreport=0,
                               pkgname="monmlp", pkgfun="monmlp.fit", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "monmlp", pkgfun = "monmlp.fit", monmlp.method,
  prepareZZ.arg = monmlp.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = TRUE)

Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "BFGS")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
BFGS
RMSE MSE MAE WAE time
mDette 0.3795 0.1440 0.2841 1.6731 0.29
mFriedman 0.0107 0.0001 0.0086 0.0358 0.31
kable(apply(res[,,2,], c(3,1), min), caption = "Nelder-Mead")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
Nelder-Mead
RMSE MSE MAE WAE time
mDette 7.1622 51.2968 5.8177 28.4143 0.29
mFriedman 0.1479 0.0219 0.1161 0.4360 0.32

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

neuralnet train/predict functions - arguments formula,data

#library(neuralnet)
neuralnet.method <- c("slr", "sag", "rprop-", "rprop+", "backprop")
hyperParams.neuralnet <- function(method, ...) {
    return(list(iter=maxit1storderD, threshold=0.1, linear.output=TRUE))
}
NNtrain.neuralnet <- function(x, y, dataxy, formula, hidden_neur, method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(method, ...))
    
    if(method == "backprop") {learningrate <- 0.001} else {learningrate <- NULL}
    
    neuralnet::neuralnet(formula = formula, data = dataxy, hidden = hidden_neur, algorithm=method,
                        threshold=hyper_params$threshold, linear.output=hyper_params$linear.output,
                                    stepmax = hyper_params$iter, startweights = NULL, act.fct = "tanh",
                                    learningrate = learningrate)
}
NNpredict.neuralnet <- function(object, x, ...)
  as.numeric(predict(object, newdata = x))

NNclose.neuralnet <- function()
  if("package:neuralnet" %in% search())
    detach("package:neuralnet", unload=TRUE)
neuralnet.prepareZZ <- list(xdmv = "d", ydmv = "d", zdm = "d", scale = TRUE)

if(FALSE)
res <- train_and_predict_1data(1, neuralnet.method, "NNtrain.neuralnet", "hyperParams.neuralnet", "NNpredict.neuralnet", 
                               NNsummary, "NNclose.neuralnet", NA, neuralnet.prepareZZ, nrep=2,
                               echo=TRUE, doplot=FALSE, echoreport=0,
                               pkgname="neuralnet", pkgfun="neuralnet", rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "neuralnet", pkgfun = "neuralnet", neuralnet.method,
  prepareZZ.arg = neuralnet.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "slr")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
slr
RMSE MSE MAE WAE time
mDette 0.4586 0.2104 0.3458 1.8852 7.35
mFriedman 0.2348 0.0551 0.1880 0.6346 15.16
kable(apply(res[,,2,], c(3,1), min), caption = "sag")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
sag
RMSE MSE MAE WAE time
mDette 8.1656 66.6770 6.5262 36.2385 14.75
mFriedman 0.2348 0.0551 0.1880 0.6346 15.81
kable(apply(res[,,3,], c(3,1), min), caption = "rprop-")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
rprop-
RMSE MSE MAE WAE time
mDette 0.4424 0.1957 0.3429 1.6280 0.6
mFriedman 0.0074 0.0001 0.0058 0.0291 2.5
kable(apply(res[,,4,], c(3,1), min), caption = "rprop+")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
rprop+
RMSE MSE MAE WAE time
mDette 0.4937 0.2437 0.3757 2.0049 0.47
mFriedman 0.0073 0.0001 0.0057 0.0280 0.83
kable(apply(res[,,5,], c(3,1), min), caption = "backprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
backprop
RMSE MSE MAE WAE time
mDette 8.1656 66.6770 6.5262 36.2385 13.73
mFriedman 0.2348 0.0551 0.1880 0.6346 14.19

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder <- 200
maxit1storder <- 1000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

nlsr train/predict functions - arguments formula,data

#library(nlsr)
nlsr.method <- "none"
hyperParams.nlsr <- function(...) {
    return (list(iter=maxit2ndorder, sdnormstart=0.1))
}
NNtrain.nlsr <- function(x, y, dataxy, formula, neur, method, hyperParams, NNfullformula, NNparam, ...) {
    
    hyper_params <- do.call(hyperParams, list(...))
    
    start <- round(rnorm(NNparam, sd = hyper_params$sdnormstart), 4)
    names(start)  <- paste0("b", 1:NNparam)
    
    nlsr::nlxb(NNfullformula, start = start, data = dataxy,
                                control = list(femax = hyper_params$iter))
}
NNpredict.nlsr <- function(object, x, ...)
  as.numeric(predict(object, x))
NNclose.nlsr <- function()
  if("package:nlsr" %in% search())
    detach("package:nlsr", unload=TRUE)
nlsr.prepareZZ <- list(xdmv = "d", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, nlsr.method, "NNtrain.nlsr", "hyperParams.nlsr", "NNpredict.nlsr", 
                               NNsummary, "NNclose.nlsr", NA, nlsr.prepareZZ, nrep=5,
                               echo=TRUE, doplot=FALSE, echoreport=0,
                               pkgname="nlsr", pkgfun="nlxb", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "nlsr", pkgfun = "nlxb", nlsr.method,
  prepareZZ.arg = nlsr.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.1462 0.0214 0.1173 0.6500 0.41
mFriedman 0.0044 0.0000 0.0036 0.0125 0.65
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.5374 0.29665 0.4295 1.95305 0.425
mFriedman 0.0215 0.00065 0.0178 0.06345 0.755

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder <- 200
maxit1storder <- 1000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

nnet Train/predict Functions - arguments x,y

#library(nnet)
nnet.method <- "none"
hyperParams.nnet <- function(...) {
    return (list(iter=maxit2ndorder, trace=FALSE))
}
NNtrain.nnet <- function(x, y, dataxy, formula, neur, method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(...))
    
    nnet::nnet(x, y, size = neur, linout = TRUE, maxit = hyper_params$iter, trace=hyper_params$trace)
}
NNpredict.nnet <- function(object, x, ...)
    predict(object, newdata=x)
NNclose.nnet <- function()
  if("package:nnet" %in% search())
    detach("package:nnet", unload=TRUE)
nnet.prepareZZ <- list(xdmv = "d", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, nnet.method, "NNtrain.nnet", "hyperParams.nnet", "NNpredict.nnet", 
                               NNsummary, "NNclose.nnet", NA, nnet.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="nnet", pkgfun="nnet", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "nnet", pkgfun = "nnet", nnet.method,
  prepareZZ.arg = nnet.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.2906 0.0845 0.2317 1.1818 0.05
mFriedman 0.0092 0.0001 0.0073 0.0285 0.08
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.60455 0.3667 0.46835 2.6066 0.07
mFriedman 0.07335 0.0054 0.05905 0.1542 0.09

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder <- 200
maxit1storder <- 1000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

qrnn Train/predict Function - arguments x,y

#library(qrnn)
qrnn.method <- "none"
hyperParams.qrnn <- function(optim_method, ...) {
    
    maxiter <- maxit2ndorder
    init.range = c(-0.1, 0.1, -0.1, 0.1)
    params <- paste0("method=", optim_method, "_iter=", maxiter)
  
    out <- list(iter = maxiter, params = params, init.range=init.range)
    return (out)
}
NNtrain.qrnn <- function(x, y, dataxy, formula, neur, optim_method, hyperParams,...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    NNreg <- qrnn::qrnn.fit(x, y, n.hidden = neur, 
                     iter.max = hyper_params$iter, n.trials = 1,
                     init.range = hyper_params$init.range, trace=FALSE)
    
    return (NNreg)
}
NNpredict.qrnn <- function(object, x, ...)
  qrnn::qrnn.predict(x, object)
NNclose.qrnn <- function()
  if("package:qrnn" %in% search())
    detach("package:qrnn", unload=TRUE)
qrnn.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, qrnn.method, "NNtrain.qrnn", "hyperParams.qrnn", "NNpredict.qrnn", 
                               NNsummary, "NNclose.qrnn", NA, qrnn.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="qrnn", pkgfun="qrnn.fit", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "qrnn", pkgfun = "qrnn.fit", qrnn.method,
  prepareZZ.arg = qrnn.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.3219 0.1036 0.2187 1.7284 0.28
mFriedman 0.0091 0.0001 0.0062 0.0304 0.28
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.4382 0.1922 0.29025 2.19145 0.630
mFriedman 0.0542 0.0041 0.04135 0.13855 0.565

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder <- 200 #predefined in package
maxit1storder <- 1000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

radiant.model Train/predict Functions - arguments xy,y

#library(radiant.model)
radiant.model.method <- "none"
hyperParams.radiant.model <- function(...) {
    return (list(type="regression", decay=0))
}
NNtrain.radiant.model <- function(x, y, dataxy, formula, neur, method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(...))
    
    radiant.model::nn(dataxy, rvar = "y", evar = attr(terms(formula), "term.labels"),
                      type = hyper_params$type, size = neur, 
                      decay = hyper_params$decay)
    
}
NNpredict.radiant.model <- function(object, x, ...)
   predict(object, pred_data=as.data.frame(x))$Prediction
NNclose.radiant.model <- function()
  if("package:radiant.model" %in% search())
    detach("package:radiant.model", unload=TRUE)
radiant.model.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, radiant.model.method, "NNtrain.radiant.model", "hyperParams.radiant.model", "NNpredict.radiant.model", 
                               NNsummary, "NNclose.radiant.model", NA, radiant.model.prepareZZ, nrep=5,
                               echo=TRUE, doplot=FALSE, echoreport=0,
                               pkgname="radiant.model", pkgfun="nn", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "radiant.model", pkgfun = "nn", radiant.model.method,
  prepareZZ.arg = radiant.model.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.0702 0.0049 0.0563 0.2745 0.20
mFriedman 0.0040 0.0000 0.0032 0.0119 0.14
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.0933 0.00885 0.07485 0.4141 0.865
mFriedman 0.0427 0.00320 0.03845 0.0804 0.480

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder <- 200
maxit1storder <- 1000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

rminer Train/predict Functions - arguments formula,data

#library(rminer)
rminer.method <- "none"
hyperParams.rminer <- function(...) {
    return (list(task="reg", iter=maxit2ndorder))
}
NNtrain.rminer <- function(x, y, dataxy, formula, neur, method, hyperParams, ...) {
    
    hyper_params <- do.call(hyperParams, list(...))
    
    rminer::fit(formula, data = dataxy, model = "mlp", task = hyper_params$task, 
                                        size = neur, maxit = hyper_params$iter)
}
NNpredict.rminer <- function(object, x, ...)
   as.numeric(rminer::predict(object, newdata=as.data.frame(x)))
NNclose.rminer <- function()
  if("package:rminer" %in% search())
    detach("package:rminer", unload=TRUE)
rminer.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, rminer.method, "NNtrain.rminer", "hyperParams.rminer", "NNpredict.rminer", 
                               NNsummary, "NNclose.rminer", NA, rminer.prepareZZ, nrep=2,
                               echo=TRUE, doplot=FALSE, echoreport=0,
                               pkgname="rminer", pkgfun="fit", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "rminer", pkgfun = "fit", rminer.method,
  prepareZZ.arg = rminer.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.2759 0.0761 0.2190 0.9034 0.22
mFriedman 0.0085 0.0001 0.0068 0.0243 0.25
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.42880 0.1839 0.32725 1.7749 0.23
mFriedman 0.01025 0.0001 0.00810 0.0329 0.28

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

RSNNS Train/predict Function - arguments x,y

#library(RSNNS)
RSNNS.method <- c("Rprop","BackpropBatch","BackpropChunk","BackpropMomentum",
                  "BackpropWeightDecay","Quickprop","SCG","Std_Backpropagation")
hyperParams.RSNNS <- function(optim_method, ...) {
    
    if(optim_method %in% c("Rprop","BackpropChunk","BackpropMomentum","BackpropWeightDecay","SCG","Std_Backpropagation"))
      maxiter <- maxit1storderA
    else
      maxiter <- maxit1storderC
    
    params <- paste0("method=", optim_method, "_iter=", maxiter)
  
    out <- list(iter = maxiter, sdnormstart=0.1)
    return (out)
}
NNtrain.RSNNS <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams, NNfullformula, NNparam,...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    start <- round(rnorm(NNparam, sd = hyper_params$sdnormstart), 4)
    names(start)  <- paste0("b", 1:NNparam)
    
    NNreg <- RSNNS::mlp(x, y, initFuncParams = start,
                 size = hidden_neur, learnFunc = optim_method, 
                 maxit = hyper_params$iter, linOut = TRUE)
    
    return (NNreg)
}
NNpredict.RSNNS <- function(object, x, ...)
  predict(object, x)
NNclose.RSNNS <- function()
  if("package:RSNNS" %in% search())
    detach("package:RSNNS", unload=TRUE)
RSNNS.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)


if(FALSE)
res <- train_and_predict_1data(1, RSNNS.method, "NNtrain.RSNNS", "hyperParams.RSNNS", "NNpredict.RSNNS", 
                               NNsummary, "NNclose.RSNNS", NA, RSNNS.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="RSNNS", pkgfun="mlp", rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "RSNNS", pkgfun = "mlp", RSNNS.method,
  prepareZZ.arg = RSNNS.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "Rprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
Rprop
RMSE MSE MAE WAE time
mDette 0.8059 0.6495 0.6172 3.011 0.62
mFriedman 0.0710 0.0050 0.0524 0.174 0.65
kable(apply(res[,,2,], c(3,1), min), caption = "BackpropBatch")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
BackpropBatch
RMSE MSE MAE WAE time
mDette 0.7212 0.5201 0.5291 4.6993 6.67
mFriedman 0.0358 0.0013 0.0267 0.1668 6.79
kable(apply(res[,,3,], c(3,1), min), caption = "BackpropChunk")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
BackpropChunk
RMSE MSE MAE WAE time
mDette 0.6379 0.4069 0.5052 2.3026 0.67
mFriedman 0.0533 0.0028 0.0424 0.1778 0.69
kable(apply(res[,,4,], c(3,1), min), caption = "BackpropMomentum")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
BackpropMomentum
RMSE MSE MAE WAE time
mDette 0.6539 0.4276 0.5004 2.6048 0.64
mFriedman 0.0653 0.0043 0.0523 0.2111 0.67
kable(apply(res[,,5,], c(3,1), min), caption = "BackpropWeightDecay")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
BackpropWeightDecay
RMSE MSE MAE WAE time
mDette 0.5845 0.3416 0.4467 2.5264 0.65
mFriedman 0.0583 0.0034 0.0465 0.2010 0.68
kable(apply(res[,,6,], c(3,1), min), caption = "Quickprop")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
Quickprop
RMSE MSE MAE WAE time
mDette 7.2233 52.1755 5.8841 28.2385 7.35
mFriedman 0.1596 0.0255 0.1241 0.5353 7.37
kable(apply(res[,,7,], c(3,1), min), caption = "SCG")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
SCG
RMSE MSE MAE WAE time
mDette 0.7608 0.5788 0.6241 2.4542 0.72
mFriedman 0.0853 0.0073 0.0763 0.1807 0.84
kable(apply(res[,,8,], c(3,1), min), caption = "Std_Backpropagation")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
Std_Backpropagation
RMSE MSE MAE WAE time
mDette 0.6578 0.4327 0.4780 2.6549 0.64
mFriedman 0.0607 0.0037 0.0473 0.2077 0.65

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 5
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

snnR Train/predict Function - arguments x,y

snnR.method <- "none"
hyperParams.snnR <- function(optim_method, ...) {
    
    maxiter <- maxit2ndorder
    params <- paste0("method=", optim_method, "_iter=", maxiter)
  
    out <- list(iter = maxiter, params = params)
    return (out)
}
NNtrain.snnR <- function(x, y, dataxy, formula, hidden_neur, optim_method, hyperParams,...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    NNreg <- snnR::snnR(x, y, nHidden = as.matrix(hidden_neur), 
                 iteramax = hyper_params$iter, verbose=FALSE)
    return (NNreg)
}
NNpredict.snnR <- function(object, x, ...)
  predict(object, x)
NNclose.snnR <- function()
  if("package:snnR" %in% search())
    detach("package:snnR", unload=TRUE)
snnR.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)


if(FALSE)
res <- train_and_predict_1data(1, simpleNeural.method, "NNtrain.snnR", "hyperParams.snnR", "NNpredict.snnR", 
                               NNsummary, "NNclose.snnR", NA, simpleNeural.prepareZZ, nrep=5, echo=TRUE, doplot=TRUE,
                               pkgname="snnR", pkgfun="snnR", rdafile=TRUE, odir=odir, echoreport=1)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "snnR", pkgfun = "snnR", snnR.method,
  prepareZZ.arg = snnR.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.8399 0.7054 0.6390 3.7672 0.05
mFriedman 0.0457 0.0021 0.0353 0.1809 0.10
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.8399 0.7054 0.6390 3.7672 0.07
mFriedman 0.0700 0.0049 0.0501 0.2113 0.13

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 2
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

TrafficBDE Train/predict Functions - arguments x,y

#library(TrafficBDE)
TrafficBDE.method <- "none"
hyperParams.TrafficBDE <- function(...) {
    return (list())
}
NNtrain.TrafficBDE <- function(x, y, dataxy, formula, neur, method, hyperParams, ...) {
  TrafficBDE::TrainCR(list(dataxy,dataxy),"y")
}
NNpredict.TrafficBDE <- function(object, x, ...)
    predict(object, x)
NNclose.TrafficBDE <- function()
  if("package:TrafficBDE" %in% search())
    detach("package:TrafficBDE", unload=TRUE)
TrafficBDE.prepareZZ <- list(xdmv = "d", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, TrafficBDE.method, "NNtrain.TrafficBDE", "hyperParams.TrafficBDE", "NNpredict.TrafficBDE", 
                               NNsummary, "NNclose.TrafficBDE", NA, TrafficBDE.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="TrafficBDE", pkgfun="TrafficBDE", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(c(1:12), pkgname = "TrafficBDE", pkgfun = "TrainCR", TrafficBDE.method,
  prepareZZ.arg = TrafficBDE.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 8.1656 66.6770 6.5262 36.2385 1646.05
mFriedman 0.2348 0.0551 0.1880 0.6346 1882.04
mIshigami 3.6898 13.6148 2.9776 13.1137 1733.67
mRef153 2.7684 7.6642 1.9895 13.3846 47.48
uDmod1 0.0411 0.0017 0.0337 0.1048 27.62
uDmod2 0.0417 0.0017 0.0341 0.0892 26.00
uDreyfus1 0.0665 0.0044 0.0454 0.2140 2.91
uDreyfus2 0.1133 0.0128 0.0894 0.2709 2.86
uGauss1 2.2405 5.0200 1.7384 7.1531 67.14
uGauss2 2.3613 5.5756 1.8489 7.5216 38.39
uGauss3 2.3033 5.3051 1.8282 7.1485 34.44
uNeuroOne 0.2430 0.0591 0.1763 0.5887 4.72
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 8.1656 66.6770 6.5262 36.2385 1646.155
mFriedman 0.2348 0.0551 0.1880 0.6346 1884.005
mIshigami 3.6898 13.6148 2.9776 13.1137 1735.165
mRef153 2.7684 7.6642 1.9895 13.3846 48.215
uDmod1 0.0411 0.0017 0.0337 0.1048 27.685
uDmod2 0.0417 0.0017 0.0341 0.0892 26.295
uDreyfus1 0.0665 0.0044 0.0454 0.2140 2.915
uDreyfus2 0.1133 0.0128 0.0894 0.2709 2.910
uGauss1 2.2405 5.0200 1.7384 7.1531 67.250
uGauss2 2.3613 5.5756 1.8489 7.5216 38.405
uGauss3 2.3033 5.3051 1.8282 7.1485 34.560
uNeuroOne 0.2430 0.0591 0.1763 0.5887 4.720

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

traineR Train/predict Functions - arguments formula

#library(traineR)
traineR.method <- "none"
hyperParams.traineR <- function(...) {
    return (list(iter=maxit2ndorder))
}
NNtrain.traineR <- function(x, y, dataxy, formula, neur, method, hyperParams, ...) {
  hyper_params <- do.call(hyperParams, list(...))
  NNreg <- traineR::train.nnet(formula = formula, data = dataxy, size = neur, maxit = hyper_params$iter, linout = TRUE)
  return(NNreg)
}
NNpredict.traineR <- function(object, x, dataxy, ...){
  object$fitted.values 
}
NNclose.traineR <- function()
  if("package:traineR" %in% search())
    detach("package:traineR", unload=TRUE)
traineR.prepareZZ <- list(xdmv = "d", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, traineR.method, "NNtrain.traineR", "hyperParams.traineR", "NNpredict.traineR", 
                               NNsummary, "NNclose.traineR", NA, traineR.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="traineR", pkgfun="traineR", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "traineR", pkgfun = "train.nnet", traineR.method,
  prepareZZ.arg = traineR.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.2566 0.0658 0.2027 1.0106 0.06
mFriedman 0.0090 0.0001 0.0071 0.0276 0.07
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.43420 0.18860 0.32990 1.87920 0.07
mFriedman 0.02795 0.00085 0.02245 0.10485 0.09

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder <- 200
maxit1storder <- 1000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

validann Train/predict Function - arguments x,y

#library(validann)
validann.method <- c("Nelder-Mead", "BFGS", "CG", "L-BFGS-B", "SANN")
hyperParams.validann <- function(optim_method, ...) {
    
    if (optim_method == "Nelder-Mead")  {maxiter <- maxit1storder} 
    if (optim_method == "BFGS")         {maxiter <- maxit2ndorder}
    if (optim_method == "CG")           {maxiter <- maxit1storder}
    if (optim_method == "L-BFGS-B")     {maxiter <- maxit2ndorder}
    if (optim_method == "SANN")         {maxiter <- maxit1storder}
    
    params <- paste0("method=", optim_method, "_iter=", maxiter)
  
    out <- list(iter = maxiter, method = optim_method, params)
    return (out)
}
NNtrain.validann <- function(x, y, dataxy, formula, neur, optim_method, hyperParams, NNfullformula, NNparam,...) {
    
    hyper_params <- do.call(hyperParams, list(optim_method, ...))
    
    iter <- hyper_params$iter
    method <- hyper_params$method
    
    NNreg <- validann::ann(x, y, size = neur, 
                           method = method, maxit = iter)
    
    return (NNreg)
}
NNpredict.validann <- function(object, x, ...)
  predict(object, x)
NNclose.validann <- function()
  if("package:validann" %in% search())
    detach("package:validann", unload=TRUE)
validann.prepareZZ <- list(xdmv = "m", ydmv = "m", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, validann.method, "NNtrain.validann", "hyperParams.validann", "NNpredict.validann", 
                               NNsummary, "NNclose.validann", NA, validann.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="validann", pkgfun="ann", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "validann", pkgfun = "ann", validann.method,
  prepareZZ.arg = validann.prepareZZ, nrep = nrep, doplot = FALSE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(apply(res[,,1,], c(3,1), min), caption = "Nelder-Mead")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
Nelder-Mead
RMSE MSE MAE WAE time
mDette 5.6452 31.8685 4.2337 25.4550 0.15
mFriedman 0.1480 0.0219 0.1146 0.4524 0.58
kable(apply(res[,,2,], c(3,1), min), caption = "BFGS")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
BFGS
RMSE MSE MAE WAE time
mDette 0.2696 0.0727 0.2149 1.0549 1.67
mFriedman 0.0096 0.0001 0.0076 0.0291 2.35
kable(apply(res[,,3,], c(3,1), min), caption = "CG")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
CG
RMSE MSE MAE WAE time
mDette 0.3878 0.1504 0.2985 1.5138 9.83
mFriedman 0.0163 0.0003 0.0129 0.0516 19.80
kable(apply(res[,,4,], c(3,1), min), caption = "L-BFGS-B")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
L-BFGS-B
RMSE MSE MAE WAE time
mDette 0.3829 0.1466 0.2950 1.5074 1.82
mFriedman 0.0143 0.0002 0.0114 0.0458 2.53
kable(apply(res[,,5,], c(3,1), min), caption = "SANN")%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
SANN
RMSE MSE MAE WAE time
mDette 3.2806 10.7625 2.4185 15.9325 0.17
mFriedman 0.1306 0.0170 0.0975 0.4036 0.18

Environment and datasets

Setup environment

library(NNbenchmark)
library(kableExtra)
options(scipen = 999)
if(dir.exists("D:/GSoC2020/Results/2020run02/"))
{  
  odir <- "D:/GSoC2020/Results/2020run02/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "~"

nrep <- 10
maxit2ndorder  <-    200
maxit1storderA <-   1000
maxit1storderB <-   5000
maxit1storderC <-  10000
maxit1storderD <- 100000

Datasets to test

NNdataSummary(NNdatasets)
##      Dataset n_rows n_inputs n_neurons n_parameters
## 1     mDette    500        3         5           26
## 2  mFriedman    500        5         5           36
## 3  mIshigami    500        3        10           51
## 4    mRef153    153        5         3           22
## 5     uDmod1     51        1         6           19
## 6     uDmod2     51        1         5           16
## 7  uDreyfus1     51        1         3           10
## 8  uDreyfus2     51        1         3           10
## 9    uGauss1    250        1         5           16
## 10   uGauss2    250        1         4           13
## 11   uGauss3    250        1         4           13
## 12 uNeuroOne     51        1         2            7

Dedicated functions by packages

yager Train/predict Function - arguments x,y

#library(yager)
yager.method <- "none"
hyperParams.yager <- function(optim_method, ...) {
    return (list(sigma = 0.15))
}
NNtrain.yager <- function(x, y, dataxy, formula, neur, optim_method, hyperParams,...) {
    
    hyper_params <- do.call(hyperParams.yager, list(yager.method))
    sigma <- hyper_params$sigma
    
    NNreg <- yager::grnn.fit(x = x, y = y, sigma = sigma)
    return (NNreg)
}
NNpredict.yager <- function(object, x, ...)
    yager::grnn.predict(object, x)
NNclose.yager <- function()
  if("package:yager" %in% search())
    detach("package:yager", unload=TRUE)
yager.prepareZZ <- list(xdmv = "m", ydmv = "v", zdm = "d", scale = TRUE)
if(FALSE)
res <- trainPredict_1data(1, yager.method, "NNtrain.yager", "hyperParams.yager", "NNpredict.yager", 
                               NNsummary, "NNclose.yager", NA, yager.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="yager", pkgfun="yager", csvfile=TRUE, rdafile=TRUE, odir=odir)
if(FALSE)
res <- trainPredict_1mth1data(1, yager.method[1], "NNtrain.yager", "hyperParams.yager", "NNpredict.yager", 
                               NNsummary, yager.prepareZZ, nrep=5, echo=TRUE, doplot=FALSE,
                               pkgname="yager", pkgfun="yager", csvfile=TRUE, rdafile=TRUE, odir=odir)

Launch the NN package

res <- trainPredict_1pkg(1:2, pkgname = "yager", pkgfun = "grnn.fit", yager.method,
  prepareZZ.arg = yager.prepareZZ, nrep = nrep, doplot = TRUE,
  csvfile = TRUE, rdafile = TRUE, odir = odir, echo = FALSE)

Results

#print(res)
kable(t(apply(res, c(1,4), min)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.4741 0.2248 0.3193 1.9227 1.04
mFriedman 0.0001 0.0000 0.0000 0.0010 1.07
kable(t(apply(res, c(1,4), median)))%>%
  kable_styling(bootstrap_options = c("striped", "hover", "condensed"))
RMSE MSE MAE WAE time
mDette 0.4741 0.2248 0.3193 1.9227 1.05
mFriedman 0.0001 0.0000 0.0000 0.0010 1.08